﻿using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Net;
using System.IO;

namespace SBLib.Common
{
    public static class CrawlWebpage
    {
        public static string GetWebpageContent(string Url)
        {
            return ProcessWebpage(Url, "");
        }

        public static string WriteWebpageContentToFile(string Url, string FilePath)
        {
            return ProcessWebpage(Url, FilePath);
        }
        private static string ProcessWebpage(string Url, string FilePath)
        {
            try
            {
                WebRequest request = WebRequest.Create(Url.Trim());
                using (WebResponse response = request.GetResponse())
                {
                    using (StreamReader responseReader = new StreamReader(response.GetResponseStream()))
                    {
                        string responseData = responseReader.ReadToEnd();

                        if (FilePath.Trim().Length > 0)
                        {
                            using (StreamWriter writer = new StreamWriter(FilePath.Trim()))
                            {
                                writer.Write(responseData);
                            }

                        }
                        return responseData;
                    }
                }

            }
            catch (Exception ex)
            {

                return ex.Message;
            }
        }

        /// <summary>
        /// Strip HTML of Tags
        /// </summary>
        /// <param name="source">HTML content</param>
        /// <returns>Plain content</returns>
        public static string StripHtml(string source)
        {
            try
            {
                string[] patterns = {
                            @"<(.|\n)*?>", //general HTML tags
                            @"<script.*?</script>" //script tags
                };
                
                string stripped = source;
                foreach (string pattern in patterns)
                {
                    stripped = System.Text.RegularExpressions.Regex.Replace(
                    stripped, pattern, string.Empty);
                }

                return stripped;
            }
            catch (Exception ex)
            {

                return ex.Message;
            }

        }
    }
}